import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;

/**
 * 利用webCollector爬取包图网上的背景音乐素材
 */
public class MusicCrawler extends BreadthCrawler{

    /**
     * 重写构造方法
     * @param crawlPath
     * @param autoParse
     */
    public MusicCrawler(String crawlPath, boolean autoParse) {
        super(crawlPath, autoParse);
        //添加种子页面
        this.addSeed("http://ibaotu.com/peiyue/11-0-0-0-0-1.html");
        //设置采集规则的类型网页
        this.addRegex("http://ibaotu.com/peiyue/11-[0-9]+-[0-9]+-[0-9]+-[0-9]+-[0-9]+\\.html");
        //设置线程数
        this.setThreads(5);
        //不匹配图片
        this.addRegex("-.*\\.(jpg|png|gif).*");
        //爬取URL上限
        getConf().setTopN(50);
    }

    public void visit(Page page, CrawlDatums next){
        String url = page.url();
        if(page.matchUrl("http://ibaotu.com/peiyue/11-[0-9]+-[0-9]+-[0-9]+-[0-9]+-[0-9]+\\.html")){
            String title = page.select("div[class=audio-info]>a").first().text();
            String audio = page.select("audio>source").attr("src");
            String endTime = page.select("span[class=end-time]").first().text();
            System.out.println("title:\n"+title);
            System.out.println("audio:\n"+audio);
            System.out.println("endTime:\n"+endTime);
            WirteToFile.out(title,audio,endTime);
        }
    }

    public static void main(String[] args) throws Exception{
        MusicCrawler musicCrawler = new MusicCrawler("crawl",true);
        //设置爬虫深度
        musicCrawler.start(3);
    }
}
